package org.jugile.util;

/*

Copyright (C) 2007-2012 Jukka Rahkonen  email: jukka.rahkonen@iki.fi

This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.

This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
Lesser General Public License for more details.

You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

*/

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.jugile.util.Jugile;

/**
 * "Be not rash with thy mouth, and let not thine heart be hasty to utter any 
 *  thing/word before God: for God is in heaven, and thou upon earth: therefore 
 *  let thy words be few. For a dream cometh through the multitude of business; 
 *  and a fool's voice is known by multitude of words." (Saarn 5:2-3)
 *  
 * ==========
 *  
 * Quick compress and uncompress utility for packing finnish and swedish texts.
 * Class includes simple ngram analyzer which can be used to tune char arrays
 * for different types of texts.
 * 
 * @author jukka.rahkonen@iki.fi
 */
public class QCompress extends Jugile {

	private final static String[] cb2 = {
		"--",// = 1193219
		"n ",// = 696147
		"a ",// = 543436
		"00",// = 447723
		"en",// = 445112
		"ta",// = 366647
		"in",// = 356319
		"an",// = 349141
		"is",// = 340812
		"st",// = 295774
		"01",// = 283971
		"le",// = 262146
		"aa",// = 248782
		"es",// = 247995
		"02",// = 238398
		"12",// = 230786
		"te",// = 222491
		"tt",// = 222422
		"on",// = 221986
		"ti",// = 216904
		"ja",// = 213644
		"20",// = 211185
		"er",// = 210248
		"et",// = 206857
		" k",// = 204439
		"ka",// = 203703
		"i ",// = 202992
		"se",// = 202012
		"si",// = 200979
		"t ",// = 198269
		"el",// = 197009
		"ll",// = 196837
		"la",// = 196029
		". ",// = 192904
		"al",// = 187450
		"-1",// = 177247
		"it",// = 175179
		"ar",// = 171266
		"ai",// = 170320
		"sa",// = 167173
		"21",// = 163249
		" s",// = 163165
		"2-",// = 162915
		"li",// = 160328
		"at",// = 156816
		"13",// = 155509
		" t",// = 155330
		"va",// = 150688
		"il",// = 150395
		"ut",// = 149070
		" o",// = 147706
		" j",// = 146345
		"ma",// = 146091
		"-0",// = 144413
		"ri",// = 143534
		"22",// = 143155
		" p",// = 141531
		"ne",// = 138369
		" 1",// = 138198
		"ä ",// = 138047
		" m",// = 136549
		"ss",// = 135423
		"tu",// = 134129
		"ki",// = 132454
		"as",// = 131413
		"ik",// = 130321
		" v",// = 128998
		"na",// = 127863
		"uu",// = 127671
		"sl",// = 127478
		".0",// = 127237
		"us",// = 124888
		"ko",// = 124434
		"11",// = 123830
		"mi",// = 122348
		"nt",// = 121950
		"LA",// = 121204
		"MA",// = 121197
		"SM",// = 121123
		"uo",// = 120801
		"AS",// = 120763
		"0-",// = 120703
		"PL",// = 120640
		"ra",// = 120026
		"to",// = 119371
		"de",// = 118827
		"10",// = 118198
		"tä",// = 117011
		"0s",// = 116755
		"s0",// = 116753
		"fi",// = 115780
		", ",// = 113562
		"un",// = 110891
		"oi",// = 109923
		"e ",// = 109710
		"me",// = 108545
		"1-",// = 107064
		"ke",// = 104741
		"ku",// = 103450
		"ii",// = 101044
		"ee",// = 100847
		"io",// = 100139
		" a",// = 100073
		"or",// = 99253
		"14",// = 98715
		"än",// = 95709
		"om",// = 95082
		"o ",// = 94680
		"nn",// = 94666
		"sk",// = 94086
		"-P",// = 93901
		"ei",// = 92877
		"kk",// = 90052
		"ol",// = 88933
		"lu",// = 88444
		"ää",// = 88210
		"r ",// = 88040
		"di",// = 87671
		"ni",// = 86732
		"vi",// = 86352
		"ad",// = 85489
		"am",// = 84243
		" e",// = 84050
		"ks",// = 83012
		"ie",// = 82286
		"A-",// = 82228
		" h",// = 81910
		"ia",// = 81487
		"s ",// = 79809
		"a.",// = 79242
		"vä",// = 79147
		" S",// = 78958
		"he",// = 78656
		"ns",// = 78201
		"i-",// = 78073
		" l",// = 76311
		"os",// = 75798
		"ul",// = 75683
		"iv",// = 74915
		"ak",// = 74025
		"mu",// = 72760
		" K",// = 71823
		"lä",// = 71415
		"im",// = 71051
		"re",// = 70611
		"pa",// = 70312
		"17",// = 70228
		"ur",// = 68408
		"uk",// = 67187
		"-f",// = 66980
		"23",// = 66961
		":1",// = 65732
		"36",// = 63963
		"ot",// = 63664
		"41",// = 63304
		"jo",// = 62633
		"ha",// = 62499
		"1:",// = 62406
		"lo",// = 61335
		"au",// = 60765
		" M",// = 60494
		"sä",// = 60179
		" R",// = 59890
		"16",// = 59877
		" T",// = 59347
		"lt",// = 59088
		"rt",// = 58634
		"30",// = 58532
		" i",// = 58344
		"av",// = 58133
		"Ra",// = 56779
		"ap",// = 55794
		"32",// = 55505
		"40",// = 55497
		"äi",// = 55436
		"31",// = 55420
		"18",// = 55156
		"ty",// = 55071
		"ve",// = 54925
		" u",// = 54788
		" n",// = 54131
//		"15",// = 54074
//		"1 ",// = 53424
//		"ht",// = 52974
//		":0",// = 52969
//		"19",// = 52901
//		"aj",// = 52821
//		"pi",// = 51551
//		"ro",// = 50212
//		"n.",// = 49986
//		"ok",// = 49845
//		"nd",// = 49595
//		"su",// = 49538
//		"24",// = 49455
//		": ",// = 48951
//		"oh",// = 48740
//		" P",// = 48523
//		"07",// = 46912
//		"no",// = 46828
//		"pe",// = 46678
//		"so",// = 46509
//		"28",// = 45890
//		"pä",// = 45855
//		" H",// = 45761		
	};
	
	private final static String[] cb1 = {
		null,
		" ",// = 2989078
		"a",// = 2771507
		"i",// = 2229567
		"-",// = 2163515
		"e",// = 2031426
		"t",// = 1874296
		"n",// = 1860279
		"s",// = 1733004
		"0",// = 1598070
		"1",// = 1317697
		"l",// = 1289465
		"o",// = 1235835
		"u",// = 1129069
		"2",// = 1103026
		"k",// = 984686
		"r",// = 884310
		"ä",// = 722423
		"m",// = 657901
		"3",// = 553269
		"4",// = 495006
		"v",// = 486228
		".",// = 485187
		"d",// = 461700
		"p",// = 402423
		"h",// = 401166
		"6",// = 395351
		"7",// = 393276
		"A",// = 391818
		"5",// = 389788
		"8",// = 388396
		"j",// = 382216
		"9",// = 370493
		"y",// = 312166
		":",// = 309527
		"f",// = 280683
		"S",// = 279670
		"L",// = 275089
		"M",// = 231618
		"P",// = 212927
		"E",// = 184297
		"b",// = 183793
		"c",// = 171029
		"g",// = 148992
		"R",// = 145566
		"T",// = 124682
		"K",// = 123792
		"ö",// = 122708
		",",// = 114531
		"N",// = 82075
		"Y",// = 71041
		"I",// = 68474
		"H",// = 66616
		"V",// = 62020
		"F",// = 57882
		"U",// = 57235
		"J",// = 55618
		"å",// = 55239
		"D",// = 50112
		"O",// = 43032
		"_",// = 41437
		"B",// = 35393
		")",// = 33007
		"(",// = 32975
		"/",// = 29515
		"C",// = 24123
		"G",// = 23212
		"w",// = 21540
		"?",// = 20005
		"x",// = 14992
		"X",// = 13227
		"W",// = 12761
		"\n",// = 11443
	};
	
	static {
		init();
	}
	
	private static String[] cb;
	private static byte offset1;
	public static void init() {
		cb = new String[256];
		cb[0] = null;
		for (byte i = -127; i < 126; i++) {
			int idx = i+128;
			String str = null;
			if (idx < cb2.length+1) str = cb2[idx-1];
			else str = cb1[idx-cb2.length];
			cb[idx] = str;
		}
		
		//print("cb:");
		for (byte i = -128; i < 127; i++) {
			//print(" " + i + " = " +cb[i+128]);
		}
		
		int off1 = -128+cb2.length+1;
		//print("off1:"+off1);
		offset1 = (byte)(-128+cb2.length+1);
		//print("offset1:"+offset1);
		//print("first 1: '" + cb[offset1+128] +"'");		
	}


	public static void main(String args[]) {
		//if (true) return;
		
		byte[] s = compress("Jukka Ra € ê â ã no \njee");
		print("size: " + s.length);
		for (int i = 0; i < s.length; i++) {
			//print("byte: " + s[i] + " :" + toStr(s[i]));
		}

		print("decompress: " + decompress(s));
		
//		resetNgram();
//		ngram("Hello World!!");
//		ngram("Hello again");
//		ngram("Jollo jalla");
//		dumpNgram();
		
	}
	
	private static int unsigned(byte b) {
		if (b >= 0) return b;
		int v = -b;
		return 128+ (128-v);
	}

	public static String decompress(byte[] s) {
		StringBuffer sb = new StringBuffer();
		for (int i = 0; i < s.length; i++) {
			byte b = s[i];
			if (b == 126) {
				byte bits[] = new byte[2];
				char ch1 = (char)unsigned(s[i+1]);
				char ch2 = (char)unsigned(s[i+2]);
				char ch = (char)(ch1*256+ch2);				
				//print("ch: " + ch + " " + (int)ch);
				i += 2;
				sb.append(ch);
				continue;
			}
			sb.append(cb[b+128]);
		}
		return sb.toString();
	}
	
	public static byte[] compress(String str) {
		byte[] res = new byte[str.length()*3];
		int size = 0;
		int len = str.length();
		for (int i = 0; i < len; ) {
			boolean found = false;
			
			for (byte j = -127; j < 126; j++) {
				//print("j="+j + " offset1="+offset1);
				if (j < offset1) {
					if (i < len-1 && str.charAt(i) == cb[j+128].charAt(0) &&
							str.charAt(i+1) == cb[j+128].charAt(1)) {
						res[size] = j;
						size++;
						i+=2;
						found = true;
						//print("2: " + cb[j+128]);
						break;
					}
				} else {
					if (str.charAt(i) == cb[j+128].charAt(0)) {
						res[size] = j;
						size++;
						i+=1;
						found = true;
						//print("1: " + cb[j+128]);
						break;
					}					
				}
			}
			if (found) continue;
			
			// not found
			res[size++] = 126;
			char ch = str.charAt(i);
			//print("not found: " + ch + " " + (int)ch);
			int ch1 = (ch&0xFF00)>>8;
			int ch2 = ch&0x00FF;
			res[size++] = (byte)ch1;
			res[size++] = (byte)ch2;
			i++;
		}
		// shrink
		byte[] s = Arrays.copyOf(res, size);
		return s;
	}
	
	private static Map<String,Integer> ngram1;
	private static Map<String,Integer> ngram2;
	private static Map<String,Integer> ngram3;
	
	public static void resetNgram() {
		ngram1 = new HashMap<String,Integer>();
		ngram2 = new HashMap<String,Integer>();
		ngram3 = new HashMap<String,Integer>();
	}
	
	public static void ngram(String str) {
		int len = str.length();
		for (int i = 0; i < len; i++) {
			String n1 = str.substring(i,i+1);
			incrNgram(ngram1,n1);
			if (i < len-1) {
				String n2 = str.substring(i,i+2);
				incrNgram(ngram2,n2);
				if (i < len-2) {
					String n3 = str.substring(i,i+3);
					incrNgram(ngram3,n3);
				}				
			}
		}
	}
	
	private static void incrNgram(Map<String,Integer> ngram, String ng) {
		int old = 0;
		if (ngram.containsKey(ng)) old = ngram.get(ng);
		old++;
		ngram.put(ng,old);
	}
	
	public static void dumpNgram() {
		int mincount = 100;
		
		List<Entry<String,Integer>> l1 = new ArrayList<Entry<String,Integer>>(); 
		List<Entry<String,Integer>> l2 = new ArrayList<Entry<String,Integer>>(); 
		List<Entry<String,Integer>> l3 = new ArrayList<Entry<String,Integer>>(); 
		
		for (Entry<String,Integer> e : ngram1.entrySet()) {
			l1.add(e);			
		}
		for (Entry<String,Integer> e : ngram2.entrySet()) {
			if (e.getValue() > mincount-1) l2.add(e);			
		}
		for (Entry<String,Integer> e : ngram3.entrySet()) {
			if (e.getValue() > mincount-1) l3.add(e);			
		}

		Collections.sort(l1, new NgramComparator());
		Collections.sort(l2, new NgramComparator());
		Collections.sort(l3, new NgramComparator());
		
		for (Entry<String,Integer> e : l3) {
			print(" " + e.getKey() + " = " + e.getValue());
		}
		print("");
		print("");
		print("========");
		print("");
		print("");
		for (Entry<String,Integer> e : l2) {
			print("\t\""+ e.getKey() +"\",// = " + e.getValue());
		}
		print("========");
		print("");
		print("");
		for (Entry<String,Integer> e : l1) {
			print("\t\""+ e.getKey() +"\",// = " + e.getValue());
		}
	}
	
	public static class NgramComparator implements Comparator<Entry<String,Integer>> {
		public int compare(Entry<String, Integer> o1, Entry<String, Integer> o2) {
			return o2.getValue().compareTo(o1.getValue());
		}
	}
	
}
